# -*- coding: utf-8 -*-
import scrapy,json,time,random
from scrapy.http import Request
from weixin_data.items import WeixinDataItem
from weixin_data.conf import get_cookie,get_conn

class ListspiderSpider(scrapy.Spider):
    name = 'listspider'
    allowed_domains = ['mp.weixin.qq.com']
    #start_urls = ['http://mp.weixin.qq.com/']

    def start_requests(self):
        self.conn = get_conn()
        cookies = get_cookie()
        url="https://mp.weixin.qq.com/cgi-bin/newmasssendpage?count=7&begin=%s&token=1229899799&lang=zh_CN&token=1229899799&lang=zh_CN&f=json&ajax=1"
        for page in range(20,51):
            print("当前页数=>%d"%(page))
            time.sleep(random.randint(5,12))
            yield Request(url%(page*7), dont_filter=True,cookies=cookies,callback=self.parse)
            

    def parse(self, response):
        content=response.text
        content=json.loads(content)
        for v in content["sent_list"]:
            timer=v["sent_info"]["time"]
            for d in v["appmsg_info"]:
                item=WeixinDataItem()
                item["title"]=d["title"]
                item["content_url"]=d["content_url"]
                item["comment_id"]=""#d["comment_id"]
                item["publish_time"]=time.strftime("%Y-%m-%d %H:%M:%S",time.gmtime(timer))
                yield item